#coding=utf-8
import scrapy
import datetime
import os
import re
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

import traceback

class VOASpider(scrapy.Spider):
	name = "voa"
	root_dir = "voa/"
	base_url = "http://www.51voa.com/"

	def __init__(self, date=None):
		if None == date:
			self.today_str, self.today = self.get_today_date()
		else:
			self.today_str = date
			self.today = self.parse_date(date)

	def start_requests(self):
		urls = [
			self.base_url,
		]
		for url in urls:
			yield scrapy.Request(url=url, callback=self.parse_index)

	def parse_index(self, response):
		# self.today_str, self.today = self.get_today_date()
		if not os.path.exists(self.root_dir+self.today_str+r"/"):
			os.mkdir(self.root_dir+self.today_str+r"/")
		total_li = len(response.css("div#list>ul>li").extract())
		for x in xrange(1, total_li+1):
			if 0 >= len(response.css("div#list>ul>li:nth-child("+str(x)+")>a")):
				continue
			date_str = response.css("div#list>ul>li:nth-child("+str(x)+")>a")[-1].css("::text").extract()[0].split("(")[-1][:-1]
			date = datetime.datetime.strptime(date_str, "%Y-%m-%d")
			if self.today == date:
				blog_link = self.base_url + response.css("div#list>ul>li:nth-child("+str(x)+")>a")[-1].css("::attr(href)").extract()[0]
				yield scrapy.Request(url=blog_link, callback=self.parse_blog, meta={"save_path": self.today_str,})

	def parse_blog(self, response):
		if 0 >= len(response.css("#mp3::attr(href)")):
			return
		mp3_link = response.css("#mp3::attr(href)").extract()[0]
		yield scrapy.Request(url=mp3_link, callback=self.download_mp3, meta={"save_path": response.meta["save_path"],})

	def download_mp3(self, response):
		save_path = response.meta["save_path"]
		mp3_url = response.url.split("?")[0]
		mp3_name = mp3_url.split("/")[-1]
		with open(self.root_dir+save_path+r"/"+mp3_name, "wb") as f:
			f.write(response.body)
		return

	def get_today_date(self):
		today_str = datetime.datetime.now().strftime("%Y-%m-%d")
		today = datetime.datetime.strptime(today_str, "%Y-%m-%d")
		return today_str, today

	def parse_date(self, datestr):
		try:
			date = datetime.datetime.strptime(datestr, "%Y-%m-%d")
		except:
			traceback.print_exc()
			sys.exit(1)
		return date
